{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<a href=\"https://colab.research.google.com/github/timeseriesAI/tsai/blob/master/tutorial_nbs/05_TS_archs_comparison.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "created by Ignacio Oguiza - email: oguiza@timeseriesAI.co"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Import libraries 📚"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "vscode": {
     "languageId": "python"
    }
   },
   "outputs": [],
   "source": [
    "# # **************** UNCOMMENT AND RUN THIS CELL IF YOU NEED TO INSTALL/ UPGRADE TSAI ****************\n",
    "# stable = True # Set to True for latest pip version or False for main branch in GitHub\n",
    "# !pip install {\"tsai -U\" if stable else \"git+https://github.com/timeseriesAI/tsai.git\"} >> /dev/null"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "vscode": {
     "languageId": "python"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "os              : Linux-5.10.133+-x86_64-with-Ubuntu-18.04-bionic\n",
      "python          : 3.7.15\n",
      "tsai            : 0.3.5\n",
      "fastai          : 2.7.10\n",
      "fastcore        : 1.5.27\n",
      "torch           : 1.12.1+cu113\n",
      "device          : 1 gpu (['Tesla T4'])\n",
      "cpu cores       : 1\n",
      "threads per cpu : 2\n",
      "RAM             : 12.68 GB\n",
      "GPU memory      : [14.75] GB\n"
     ]
    }
   ],
   "source": [
    "from tsai.all import *\n",
    "from IPython.display import clear_output\n",
    "my_setup()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Experiments 🧪"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "I've run a small test to show how you can build any model in the `tsai` library using the `create_model` functions.\n",
    "\n",
    "The esiest way to do it is pass the architecture you want to use, a `TSDataLoaders` and any `kwargs` you'd like to use. The `create_model` function will pick up the necessary data from the `TSDataLoaders` object.\n",
    "\n",
    "I've used 2 multivariate time series datasets. As you can see, it's difficult to predict which architecture will work best for which dataset.\n",
    "\n",
    "Please, bear in mind that this is a very simple test without any hyperparameter tuning."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "vscode": {
     "languageId": "python"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "  <div id=\"df-60bf9641-f5bc-40fa-b452-6c6d5de6617f\">\n",
       "    <div class=\"colab-df-container\">\n",
       "      <div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>arch</th>\n",
       "      <th>hyperparams</th>\n",
       "      <th>total params</th>\n",
       "      <th>train loss</th>\n",
       "      <th>valid loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>LSTM_FCN</td>\n",
       "      <td>{}</td>\n",
       "      <td>347246</td>\n",
       "      <td>0.074354</td>\n",
       "      <td>0.116819</td>\n",
       "      <td>0.966667</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>FCN</td>\n",
       "      <td>{}</td>\n",
       "      <td>285446</td>\n",
       "      <td>0.074158</td>\n",
       "      <td>0.125400</td>\n",
       "      <td>0.961111</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>LSTM_FCN</td>\n",
       "      <td>{'shuffle': False}</td>\n",
       "      <td>336446</td>\n",
       "      <td>0.066596</td>\n",
       "      <td>0.113484</td>\n",
       "      <td>0.961111</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>XceptionTime</td>\n",
       "      <td>{}</td>\n",
       "      <td>403420</td>\n",
       "      <td>0.399169</td>\n",
       "      <td>0.499384</td>\n",
       "      <td>0.961111</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ResNet</td>\n",
       "      <td>{}</td>\n",
       "      <td>490758</td>\n",
       "      <td>0.025244</td>\n",
       "      <td>0.122984</td>\n",
       "      <td>0.955556</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>ResCNN</td>\n",
       "      <td>{}</td>\n",
       "      <td>268551</td>\n",
       "      <td>0.041432</td>\n",
       "      <td>0.134848</td>\n",
       "      <td>0.950000</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>InceptionTime</td>\n",
       "      <td>{}</td>\n",
       "      <td>460038</td>\n",
       "      <td>0.026669</td>\n",
       "      <td>0.113258</td>\n",
       "      <td>0.938889</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>mWDN</td>\n",
       "      <td>{'levels': 4}</td>\n",
       "      <td>467038</td>\n",
       "      <td>0.025454</td>\n",
       "      <td>0.274648</td>\n",
       "      <td>0.927778</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>xresnet1d34</td>\n",
       "      <td>{}</td>\n",
       "      <td>7232518</td>\n",
       "      <td>0.022142</td>\n",
       "      <td>0.318501</td>\n",
       "      <td>0.916667</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>OmniScaleCNN</td>\n",
       "      <td>{}</td>\n",
       "      <td>5239596</td>\n",
       "      <td>0.193854</td>\n",
       "      <td>0.260377</td>\n",
       "      <td>0.883333</td>\n",
       "      <td>19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>LSTM</td>\n",
       "      <td>{'n_layers': 3, 'bidirectional': True}</td>\n",
       "      <td>585206</td>\n",
       "      <td>0.353111</td>\n",
       "      <td>0.354646</td>\n",
       "      <td>0.833333</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>LSTM</td>\n",
       "      <td>{'n_layers': 2, 'bidirectional': True}</td>\n",
       "      <td>343606</td>\n",
       "      <td>0.322159</td>\n",
       "      <td>0.342655</td>\n",
       "      <td>0.822222</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>LSTM</td>\n",
       "      <td>{'n_layers': 2, 'bidirectional': False}</td>\n",
       "      <td>131806</td>\n",
       "      <td>0.365258</td>\n",
       "      <td>0.336271</td>\n",
       "      <td>0.816667</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>LSTM</td>\n",
       "      <td>{'n_layers': 1, 'bidirectional': True}</td>\n",
       "      <td>102006</td>\n",
       "      <td>0.386917</td>\n",
       "      <td>0.368431</td>\n",
       "      <td>0.816667</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>LSTM</td>\n",
       "      <td>{'n_layers': 1, 'bidirectional': False}</td>\n",
       "      <td>51006</td>\n",
       "      <td>0.379870</td>\n",
       "      <td>0.429362</td>\n",
       "      <td>0.800000</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>LSTM</td>\n",
       "      <td>{'n_layers': 3, 'bidirectional': False}</td>\n",
       "      <td>212606</td>\n",
       "      <td>0.581518</td>\n",
       "      <td>0.637237</td>\n",
       "      <td>0.716667</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>\n",
       "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-60bf9641-f5bc-40fa-b452-6c6d5de6617f')\"\n",
       "              title=\"Convert this dataframe to an interactive table.\"\n",
       "              style=\"display:none;\">\n",
       "        \n",
       "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
       "       width=\"24px\">\n",
       "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
       "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
       "  </svg>\n",
       "      </button>\n",
       "      \n",
       "  <style>\n",
       "    .colab-df-container {\n",
       "      display:flex;\n",
       "      flex-wrap:wrap;\n",
       "      gap: 12px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert {\n",
       "      background-color: #E8F0FE;\n",
       "      border: none;\n",
       "      border-radius: 50%;\n",
       "      cursor: pointer;\n",
       "      display: none;\n",
       "      fill: #1967D2;\n",
       "      height: 32px;\n",
       "      padding: 0 0 0 0;\n",
       "      width: 32px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert:hover {\n",
       "      background-color: #E2EBFA;\n",
       "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "      fill: #174EA6;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert {\n",
       "      background-color: #3B4455;\n",
       "      fill: #D2E3FC;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert:hover {\n",
       "      background-color: #434B5C;\n",
       "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
       "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
       "      fill: #FFFFFF;\n",
       "    }\n",
       "  </style>\n",
       "\n",
       "      <script>\n",
       "        const buttonEl =\n",
       "          document.querySelector('#df-60bf9641-f5bc-40fa-b452-6c6d5de6617f button.colab-df-convert');\n",
       "        buttonEl.style.display =\n",
       "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "\n",
       "        async function convertToInteractive(key) {\n",
       "          const element = document.querySelector('#df-60bf9641-f5bc-40fa-b452-6c6d5de6617f');\n",
       "          const dataTable =\n",
       "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
       "                                                     [key], {});\n",
       "          if (!dataTable) return;\n",
       "\n",
       "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
       "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
       "            + ' to learn more about interactive tables.';\n",
       "          element.innerHTML = '';\n",
       "          dataTable['output_type'] = 'display_data';\n",
       "          await google.colab.output.renderOutput(dataTable, element);\n",
       "          const docLink = document.createElement('div');\n",
       "          docLink.innerHTML = docLinkHtml;\n",
       "          element.appendChild(docLink);\n",
       "        }\n",
       "      </script>\n",
       "    </div>\n",
       "  </div>\n",
       "  "
      ],
      "text/plain": [
       "             arch                              hyperparams total params  \\\n",
       "0        LSTM_FCN                                       {}       347246   \n",
       "1             FCN                                       {}       285446   \n",
       "2        LSTM_FCN                       {'shuffle': False}       336446   \n",
       "3    XceptionTime                                       {}       403420   \n",
       "4          ResNet                                       {}       490758   \n",
       "5          ResCNN                                       {}       268551   \n",
       "6   InceptionTime                                       {}       460038   \n",
       "7            mWDN                            {'levels': 4}       467038   \n",
       "8     xresnet1d34                                       {}      7232518   \n",
       "9    OmniScaleCNN                                       {}      5239596   \n",
       "10           LSTM   {'n_layers': 3, 'bidirectional': True}       585206   \n",
       "11           LSTM   {'n_layers': 2, 'bidirectional': True}       343606   \n",
       "12           LSTM  {'n_layers': 2, 'bidirectional': False}       131806   \n",
       "13           LSTM   {'n_layers': 1, 'bidirectional': True}       102006   \n",
       "14           LSTM  {'n_layers': 1, 'bidirectional': False}        51006   \n",
       "15           LSTM  {'n_layers': 3, 'bidirectional': False}       212606   \n",
       "\n",
       "    train loss  valid loss  accuracy time  \n",
       "0     0.074354    0.116819  0.966667    6  \n",
       "1     0.074158    0.125400  0.961111    5  \n",
       "2     0.066596    0.113484  0.961111    6  \n",
       "3     0.399169    0.499384  0.961111    9  \n",
       "4     0.025244    0.122984  0.955556    7  \n",
       "5     0.041432    0.134848  0.950000    6  \n",
       "6     0.026669    0.113258  0.938889    9  \n",
       "7     0.025454    0.274648  0.927778    9  \n",
       "8     0.022142    0.318501  0.916667   15  \n",
       "9     0.193854    0.260377  0.883333   19  \n",
       "10    0.353111    0.354646  0.833333    5  \n",
       "11    0.322159    0.342655  0.822222    5  \n",
       "12    0.365258    0.336271  0.816667    4  \n",
       "13    0.386917    0.368431  0.816667    4  \n",
       "14    0.379870    0.429362  0.800000    5  \n",
       "15    0.581518    0.637237  0.716667    5  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "dsid = 'NATOPS' \n",
    "bs = 64\n",
    "X, y, splits = get_UCR_data(dsid, return_split=False)\n",
    "print(X.shape)\n",
    "tfms  = [None, [Categorize()]]\n",
    "dsets = TSDatasets(X, y, tfms=tfms, splits=splits)\n",
    "dls   = TSDataLoaders.from_dsets(dsets.train, dsets.valid, bs=[bs, bs*2])\n",
    "\n",
    "archs = [(FCN, {}), (ResNet, {}), (xresnet1d34, {}), (ResCNN, {}), \n",
    "         (LSTM, {'n_layers':1, 'bidirectional': False}), (LSTM, {'n_layers':2, 'bidirectional': False}), (LSTM, {'n_layers':3, 'bidirectional': False}), \n",
    "         (LSTM, {'n_layers':1, 'bidirectional': True}), (LSTM, {'n_layers':2, 'bidirectional': True}), (LSTM, {'n_layers':3, 'bidirectional': True}),\n",
    "         (LSTM_FCN, {}), (LSTM_FCN, {'shuffle': False}), (InceptionTime, {}), (XceptionTime, {}), (OmniScaleCNN, {}), (mWDN, {'levels': 4})]\n",
    "\n",
    "results = pd.DataFrame(columns=['arch', 'hyperparams', 'total params', 'train loss', 'valid loss', 'accuracy', 'time'])\n",
    "for i, (arch, k) in enumerate(archs):\n",
    "    model = create_model(arch, dls=dls, **k)\n",
    "    print(model.__class__.__name__)\n",
    "    learn = Learner(dls, model,  metrics=accuracy)\n",
    "    start = time.time()\n",
    "    learn.fit_one_cycle(100, 1e-3)\n",
    "    elapsed = time.time() - start\n",
    "    vals = learn.recorder.values[-1]\n",
    "    results.loc[i] = [arch.__name__, k, count_parameters(model), vals[0], vals[1], vals[2], int(elapsed)]\n",
    "    results.sort_values(by='accuracy', ascending=False, kind='stable', ignore_index=True, inplace=True)\n",
    "    clear_output()\n",
    "    display(results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "vscode": {
     "languageId": "python"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "  <div id=\"df-d3bcb35b-b955-4979-9f76-b4684a9449f9\">\n",
       "    <div class=\"colab-df-container\">\n",
       "      <div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>arch</th>\n",
       "      <th>hyperparams</th>\n",
       "      <th>total params</th>\n",
       "      <th>train loss</th>\n",
       "      <th>valid loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>XceptionTime</td>\n",
       "      <td>{}</td>\n",
       "      <td>401580</td>\n",
       "      <td>0.070966</td>\n",
       "      <td>1.596494</td>\n",
       "      <td>0.680049</td>\n",
       "      <td>92</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ResCNN</td>\n",
       "      <td>{}</td>\n",
       "      <td>260367</td>\n",
       "      <td>0.231670</td>\n",
       "      <td>1.220587</td>\n",
       "      <td>0.677210</td>\n",
       "      <td>56</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>LSTM</td>\n",
       "      <td>{'n_layers': 3, 'bidirectional': True}</td>\n",
       "      <td>572414</td>\n",
       "      <td>0.000120</td>\n",
       "      <td>2.694554</td>\n",
       "      <td>0.669100</td>\n",
       "      <td>50</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>LSTM_FCN</td>\n",
       "      <td>{'shuffle': False}</td>\n",
       "      <td>314950</td>\n",
       "      <td>0.416285</td>\n",
       "      <td>1.403729</td>\n",
       "      <td>0.662206</td>\n",
       "      <td>45</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>LSTM</td>\n",
       "      <td>{'n_layers': 2, 'bidirectional': True}</td>\n",
       "      <td>330814</td>\n",
       "      <td>0.000262</td>\n",
       "      <td>2.415106</td>\n",
       "      <td>0.657745</td>\n",
       "      <td>44</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>ResNet</td>\n",
       "      <td>{}</td>\n",
       "      <td>482574</td>\n",
       "      <td>0.021112</td>\n",
       "      <td>1.656618</td>\n",
       "      <td>0.653690</td>\n",
       "      <td>71</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>LSTM</td>\n",
       "      <td>{'n_layers': 2, 'bidirectional': False}</td>\n",
       "      <td>125414</td>\n",
       "      <td>0.002257</td>\n",
       "      <td>2.448107</td>\n",
       "      <td>0.639092</td>\n",
       "      <td>35</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>LSTM</td>\n",
       "      <td>{'n_layers': 3, 'bidirectional': False}</td>\n",
       "      <td>206214</td>\n",
       "      <td>0.001700</td>\n",
       "      <td>2.582463</td>\n",
       "      <td>0.632198</td>\n",
       "      <td>39</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>LSTM</td>\n",
       "      <td>{'n_layers': 1, 'bidirectional': True}</td>\n",
       "      <td>89214</td>\n",
       "      <td>0.056981</td>\n",
       "      <td>1.748464</td>\n",
       "      <td>0.606650</td>\n",
       "      <td>34</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>LSTM</td>\n",
       "      <td>{'n_layers': 1, 'bidirectional': False}</td>\n",
       "      <td>44614</td>\n",
       "      <td>0.058143</td>\n",
       "      <td>1.877555</td>\n",
       "      <td>0.601379</td>\n",
       "      <td>31</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>LSTM_FCN</td>\n",
       "      <td>{}</td>\n",
       "      <td>326950</td>\n",
       "      <td>0.460975</td>\n",
       "      <td>1.571851</td>\n",
       "      <td>0.592863</td>\n",
       "      <td>46</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>OmniScaleCNN</td>\n",
       "      <td>{}</td>\n",
       "      <td>1432716</td>\n",
       "      <td>0.674799</td>\n",
       "      <td>1.878544</td>\n",
       "      <td>0.554745</td>\n",
       "      <td>89</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>FCN</td>\n",
       "      <td>{}</td>\n",
       "      <td>270350</td>\n",
       "      <td>0.833404</td>\n",
       "      <td>1.675762</td>\n",
       "      <td>0.535280</td>\n",
       "      <td>40</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>mWDN</td>\n",
       "      <td>{'levels': 4}</td>\n",
       "      <td>461182</td>\n",
       "      <td>0.000141</td>\n",
       "      <td>2.827999</td>\n",
       "      <td>0.512571</td>\n",
       "      <td>111</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>InceptionTime</td>\n",
       "      <td>{}</td>\n",
       "      <td>457614</td>\n",
       "      <td>0.260041</td>\n",
       "      <td>3.254463</td>\n",
       "      <td>0.422952</td>\n",
       "      <td>100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>xresnet1d34</td>\n",
       "      <td>{}</td>\n",
       "      <td>7234894</td>\n",
       "      <td>0.067009</td>\n",
       "      <td>4.703222</td>\n",
       "      <td>0.366586</td>\n",
       "      <td>174</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>\n",
       "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-d3bcb35b-b955-4979-9f76-b4684a9449f9')\"\n",
       "              title=\"Convert this dataframe to an interactive table.\"\n",
       "              style=\"display:none;\">\n",
       "        \n",
       "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
       "       width=\"24px\">\n",
       "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
       "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
       "  </svg>\n",
       "      </button>\n",
       "      \n",
       "  <style>\n",
       "    .colab-df-container {\n",
       "      display:flex;\n",
       "      flex-wrap:wrap;\n",
       "      gap: 12px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert {\n",
       "      background-color: #E8F0FE;\n",
       "      border: none;\n",
       "      border-radius: 50%;\n",
       "      cursor: pointer;\n",
       "      display: none;\n",
       "      fill: #1967D2;\n",
       "      height: 32px;\n",
       "      padding: 0 0 0 0;\n",
       "      width: 32px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert:hover {\n",
       "      background-color: #E2EBFA;\n",
       "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "      fill: #174EA6;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert {\n",
       "      background-color: #3B4455;\n",
       "      fill: #D2E3FC;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert:hover {\n",
       "      background-color: #434B5C;\n",
       "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
       "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
       "      fill: #FFFFFF;\n",
       "    }\n",
       "  </style>\n",
       "\n",
       "      <script>\n",
       "        const buttonEl =\n",
       "          document.querySelector('#df-d3bcb35b-b955-4979-9f76-b4684a9449f9 button.colab-df-convert');\n",
       "        buttonEl.style.display =\n",
       "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "\n",
       "        async function convertToInteractive(key) {\n",
       "          const element = document.querySelector('#df-d3bcb35b-b955-4979-9f76-b4684a9449f9');\n",
       "          const dataTable =\n",
       "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
       "                                                     [key], {});\n",
       "          if (!dataTable) return;\n",
       "\n",
       "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
       "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
       "            + ' to learn more about interactive tables.';\n",
       "          element.innerHTML = '';\n",
       "          dataTable['output_type'] = 'display_data';\n",
       "          await google.colab.output.renderOutput(dataTable, element);\n",
       "          const docLink = document.createElement('div');\n",
       "          docLink.innerHTML = docLinkHtml;\n",
       "          element.appendChild(docLink);\n",
       "        }\n",
       "      </script>\n",
       "    </div>\n",
       "  </div>\n",
       "  "
      ],
      "text/plain": [
       "             arch                              hyperparams total params  \\\n",
       "0    XceptionTime                                       {}       401580   \n",
       "1          ResCNN                                       {}       260367   \n",
       "2            LSTM   {'n_layers': 3, 'bidirectional': True}       572414   \n",
       "3        LSTM_FCN                       {'shuffle': False}       314950   \n",
       "4            LSTM   {'n_layers': 2, 'bidirectional': True}       330814   \n",
       "5          ResNet                                       {}       482574   \n",
       "6            LSTM  {'n_layers': 2, 'bidirectional': False}       125414   \n",
       "7            LSTM  {'n_layers': 3, 'bidirectional': False}       206214   \n",
       "8            LSTM   {'n_layers': 1, 'bidirectional': True}        89214   \n",
       "9            LSTM  {'n_layers': 1, 'bidirectional': False}        44614   \n",
       "10       LSTM_FCN                                       {}       326950   \n",
       "11   OmniScaleCNN                                       {}      1432716   \n",
       "12            FCN                                       {}       270350   \n",
       "13           mWDN                            {'levels': 4}       461182   \n",
       "14  InceptionTime                                       {}       457614   \n",
       "15    xresnet1d34                                       {}      7234894   \n",
       "\n",
       "    train loss  valid loss  accuracy time  \n",
       "0     0.070966    1.596494  0.680049   92  \n",
       "1     0.231670    1.220587  0.677210   56  \n",
       "2     0.000120    2.694554  0.669100   50  \n",
       "3     0.416285    1.403729  0.662206   45  \n",
       "4     0.000262    2.415106  0.657745   44  \n",
       "5     0.021112    1.656618  0.653690   71  \n",
       "6     0.002257    2.448107  0.639092   35  \n",
       "7     0.001700    2.582463  0.632198   39  \n",
       "8     0.056981    1.748464  0.606650   34  \n",
       "9     0.058143    1.877555  0.601379   31  \n",
       "10    0.460975    1.571851  0.592863   46  \n",
       "11    0.674799    1.878544  0.554745   89  \n",
       "12    0.833404    1.675762  0.535280   40  \n",
       "13    0.000141    2.827999  0.512571  111  \n",
       "14    0.260041    3.254463  0.422952  100  \n",
       "15    0.067009    4.703222  0.366586  174  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "dsid = 'LSST' \n",
    "bs = 64\n",
    "X, y, splits = get_UCR_data(dsid, return_split=False)\n",
    "print(X.shape)\n",
    "tfms  = [None, [Categorize()]]\n",
    "dsets = TSDatasets(X, y, tfms=tfms, splits=splits)\n",
    "dls   = TSDataLoaders.from_dsets(dsets.train, dsets.valid, bs=[bs, bs*2])\n",
    "\n",
    "archs = [(FCN, {}), (ResNet, {}), (xresnet1d34, {}), (ResCNN, {}), \n",
    "         (LSTM, {'n_layers':1, 'bidirectional': False}), (LSTM, {'n_layers':2, 'bidirectional': False}), (LSTM, {'n_layers':3, 'bidirectional': False}), \n",
    "         (LSTM, {'n_layers':1, 'bidirectional': True}), (LSTM, {'n_layers':2, 'bidirectional': True}), (LSTM, {'n_layers':3, 'bidirectional': True}),\n",
    "         (LSTM_FCN, {}), (LSTM_FCN, {'shuffle': False}), (InceptionTime, {}), (XceptionTime, {}), (OmniScaleCNN, {}), (mWDN, {'levels': 4})]\n",
    "\n",
    "results = pd.DataFrame(columns=['arch', 'hyperparams', 'total params', 'train loss', 'valid loss', 'accuracy', 'time'])\n",
    "for i, (arch, k) in enumerate(archs):\n",
    "    model = create_model(arch, dls=dls, **k)\n",
    "    print(model.__class__.__name__)\n",
    "    learn = Learner(dls, model,  metrics=accuracy)\n",
    "    start = time.time()\n",
    "    learn.fit_one_cycle(100, 1e-3)\n",
    "    elapsed = time.time() - start\n",
    "    vals = learn.recorder.values[-1]\n",
    "    results.loc[i] = [arch.__name__, k, count_parameters(model), vals[0], vals[1], vals[2], int(elapsed)]\n",
    "    results.sort_values(by='accuracy', ascending=False, kind='stable', ignore_index=True, inplace=True)\n",
    "    clear_output()\n",
    "    display(results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "vscode": {
     "languageId": "python"
    }
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}
